{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Character based RNN language model trained on 'The Complete Works of William Shakespeare'\n",
    "Based on http://karpathy.github.io/2015/05/21/rnn-effectiveness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "RNNTYPE = :lstm\n",
    "BATCHSIZE = 64\n",
    "SEQLENGTH = 64\n",
    "INPUTSIZE = 512\n",
    "VOCABSIZE = 128\n",
    "HIDDENSIZE = 512\n",
    "NUMLAYERS = 2\n",
    "DROPOUT = 0.0\n",
    "LR=0.001\n",
    "BETA_1=0.9\n",
    "BETA_2=0.999\n",
    "EPS=1e-08\n",
    "EPOCHS = 30;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4669605"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base = joinpath(Base.JULIA_HOME, Base.DATAROOTDIR, \"julia\", \"base\")\n",
    "text = \"\"\n",
    "for (root,dirs,files) in walkdir(base)\n",
    "    for f in files\n",
    "        f[end-2:end] == \".jl\" || continue\n",
    "        text *= readstring(joinpath(root,f))\n",
    "    end\n",
    "    # println((root,length(files),all(f->contains(f,\".jl\"),files)))\n",
    "end\n",
    "length(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3247×2 Array{Any,2}:\n",
       " ' '   1006754\n",
       " 'e'    285781\n",
       " 't'    224363\n",
       " 'r'    191243\n",
       " 'n'    188651\n",
       " 'i'    180279\n",
       " 'a'    168744\n",
       " 'o'    151484\n",
       " 's'    150282\n",
       " '\\n'   137794\n",
       " 'l'    117761\n",
       " ','    101460\n",
       " 'd'     94018\n",
       " ⋮            \n",
       " '‰'         1\n",
       " '🚇'         1\n",
       " 'ˢ'         1\n",
       " '𝕴'         1\n",
       " '⁉'         1\n",
       " '𝚗'         1\n",
       " '🈲'         1\n",
       " '🔽'         1\n",
       " '≦'         1\n",
       " '⋟'         1\n",
       " '🚊'         1\n",
       " '𝖆'         1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "charcnt = Dict{Char,Int}()\n",
    "for c in text; charcnt[c]=1+get(charcnt,c,0); end\n",
    "chars = sort(collect(keys(charcnt)), by=(x->charcnt[x]), rev=true)\n",
    "charid = Dict{Char,Int}()\n",
    "for i=1:length(chars); charid[chars[i]]=i; end\n",
    "hcat(chars, map(c->charcnt[c],chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4669605, 4145317, 524288)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = map(c->charid[c], collect(text))\n",
    "data[data .> VOCABSIZE] = VOCABSIZE\n",
    "ntst = 1<<19\n",
    "tst = data[1:ntst]\n",
    "trn = data[1+ntst:end]\n",
    "length.((data,trn,tst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Load 'The Complete Works of William Shakespeare'\n",
    "using Knet\n",
    "#include(Knet.dir(\"data\",\"gutenberg.jl\"))\n",
    "#trn,tst,chars = shakespeare()\n",
    "#map(summary,(trn,tst,chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [],
   "source": [
    "# Print a sample\n",
    "r = rand(1:(length(trn)-1000))\n",
    "println(string(chars[trn[r:r+1000]]...)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1012, 127)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "# Get ambiguous method if try to unify AbstractVector/AbstractMatrix here using AbstractVecOrMat\n",
      "A_mul_B!(out::AbstractVector, A::Diagonal, in::AbstractVector) = out .= A.diag .* in\n",
      "Ac_mul_B!(out::AbstractVector, A::Diagonal, in::AbstractVector) = out .= ctranspose.(A.diag) .* in\n",
      "At_mul_B!(out::AbstractVector, A::Diagonal, in::AbstractVector) = out .= transpose.(A.diag) .* in\n",
      "\n",
      "A_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= A.diag .* in\n",
      "Ac_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= ctranspose.(A.diag) .* in\n",
      "At_mul_B!(out::AbstractMatrix, A::Diagonal, in::AbstractMatrix) = out .= transpose.(A.diag) .* in\n",
      "\n",
      "\n",
      "(/)(Da::Diagonal, Db::Diagonal) = Diagonal(Da.diag ./ Db.diag)\n",
      "function A_ldiv_B!(D::Diagonal{T}, v::AbstractVector{T}) where T\n",
      "    if length(v) != length(D.diag)\n",
      "        throw(DimensionMismatch(\"diagonal matrix is $(length(D.diag)) by $(length(D.diag)) but right hand side has $(length(v)) rows\"))\n",
      "    end\n",
      "    for i=1:length(D.diag)\n",
      " \n"
     ]
    }
   ],
   "source": [
    "# Minibatch data\n",
    "function mb(a)\n",
    "    N = div(length(a),BATCHSIZE)\n",
    "    x = reshape(a[1:N*BATCHSIZE],N,BATCHSIZE)' # reshape full data to (B,N) with contiguous rows\n",
    "    minibatch(x[:,1:N-1], x[:,2:N], SEQLENGTH) # split into (B,T) blocks \n",
    "end\n",
    "dtrn,dtst = mb(trn),mb(tst)\n",
    "map(length, (dtrn,dtst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Define model\n",
    "function initmodel()\n",
    "    w(d...)=KnetArray(xavier(Float32,d...))\n",
    "    b(d...)=KnetArray(zeros(Float32,d...))\n",
    "    r,wr = rnninit(INPUTSIZE,HIDDENSIZE,rnnType=RNNTYPE,numLayers=NUMLAYERS,dropout=DROPOUT)\n",
    "    wx = w(INPUTSIZE,VOCABSIZE)\n",
    "    wy = w(VOCABSIZE,HIDDENSIZE)\n",
    "    by = b(VOCABSIZE,1)\n",
    "    return r,wr,wx,wy,by\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predict (generic function with 1 method)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Given the current character, predict the next character\n",
    "function predict(ws,xs,hx,cx;pdrop=0)\n",
    "    r,wr,wx,wy,by = ws\n",
    "    x = wx[:,xs]                                    # xs=(B,T) x=(X,B,T)\n",
    "    x = dropout(x,pdrop)\n",
    "    y,hy,cy = rnnforw(r,wr,x,hx,cx,hy=true,cy=true) # y=(H,B,T) hy=cy=(H,B,L)\n",
    "    y = dropout(y,pdrop)\n",
    "    y2 = reshape(y,size(y,1),size(y,2)*size(y,3))   # y2=(H,B*T)\n",
    "    return wy*y2.+by, hy, cy\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Define loss and its gradient\n",
    "function loss(w,x,y,h;o...)\n",
    "    py,hy,cy = predict(w,x,h...;o...)\n",
    "    h[1],h[2] = getval(hy),getval(cy)\n",
    "    return nll(py,y)\n",
    "end\n",
    "\n",
    "lossgradient = gradloss(loss);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "function train(model,data,optim)\n",
    "    hiddens = Any[nothing,nothing]\n",
    "    Σ,N=0,0\n",
    "    for (x,y) in data\n",
    "        grads,loss1 = lossgradient(model,x,y,hiddens;pdrop=DROPOUT)\n",
    "        update!(model, grads, optim)\n",
    "        Σ,N=Σ+loss1,N+1\n",
    "    end\n",
    "    return Σ/N\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "function test(model,data)\n",
    "    hiddens = Any[nothing,nothing]\n",
    "    Σ,N=0,0\n",
    "    for (x,y) in data\n",
    "        Σ,N = Σ+loss(model,x,y,hiddens), N+1\n",
    "    end\n",
    "    return Σ/N\n",
    "end; "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# Sample from trained model\n",
    "function generate(model,n)\n",
    "    function sample(y)\n",
    "        p,r=Array(exp.(y-logsumexp(y))),rand()\n",
    "        for j=1:length(p); (r -= p[j]) < 0 && return j; end\n",
    "    end\n",
    "    h,c = nothing,nothing\n",
    "    chars = model[end]\n",
    "    x = findfirst(chars,'\\n')\n",
    "    for i=1:n\n",
    "        y,h,c = predict(model,[x],h,c)\n",
    "        x = sample(y)\n",
    "        print(chars[x])\n",
    "    end\n",
    "    println()\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false,
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "#=\n",
    "# Train model or load from file if exists\n",
    "using JLD\n",
    "model=optim=nothing; knetgc()\n",
    "if true # !isfile(\"juliacharlm.jld\")\n",
    "    model = initmodel()\n",
    "    optim = optimizers(model, Adam; lr=LR, beta1=BETA_1, beta2=BETA_2, eps=EPS);    info(\"Training...\")\n",
    "    @time for epoch in 1:EPOCHS\n",
    "        @time trnloss = train(model,dtrn,optim) # ~18 seconds\n",
    "        @time tstloss = test(model,dtst)        # ~0.5 seconds\n",
    "        println((:epoch, epoch, :trnppl, exp(trnloss), :tstppl, exp(tstloss)))\n",
    "    end\n",
    "    save(\"juliacharlm.jld\",\"model\",model)\n",
    "else\n",
    "    model = load(\"juliacharlm.jld\",\"model\")\n",
    "end\n",
    "summary(model)\n",
    "=#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# generate(model,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "hyperloss (generic function with 1 method)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function hyperloss(v)\n",
    "    global HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, EPOCHS, model\n",
    "    HIDDENSIZE = floor(Int, 128 * 2^v[1])\n",
    "    INPUTSIZE = floor(Int, 128 * 2^v[2])\n",
    "    NUMLAYERS = max(1,floor(Int,v[3]))\n",
    "    DROPOUT = isa(v[4],Number) ? sigm(v[4]) : 0\n",
    "    @show (HIDDENSIZE,INPUTSIZE,NUMLAYERS,DROPOUT)\n",
    "    knetgc()\n",
    "    model = initmodel()\n",
    "    optim = optimizers(model, Adam; lr=LR, beta1=BETA_1, beta2=BETA_2, eps=EPS)\n",
    "    for epoch in 1:10\n",
    "        trnloss = train(model,dtrn,optim) # ~18 seconds\n",
    "        tstloss = test(model,dtst)        # ~0.5 seconds\n",
    "        println((:epoch, epoch, :trnppl, exp(trnloss), :tstppl, exp(tstloss)))\n",
    "    end\n",
    "    trnloss = test(model,dtrn)\n",
    "    trnppl = exp(trnloss)\n",
    "    @show (HIDDENSIZE,INPUTSIZE,NUMLAYERS,DROPOUT,trnppl)\n",
    "    return trnppl\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mcudnn.cudnnRNNForwardInference error 8\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mcudnn.cudnnRNNForwardInference error 8\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/.julia/v0.6/Knet/src/gpu.jl:18\u001b[22m\u001b[22m [inlined]",
      " [2] \u001b[1m#rnnforw#487\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Ptr{Void}, ::Bool, ::Void, ::Bool, ::Bool, ::Function, ::Knet.RNN, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/.julia/v0.6/Knet/src/rnn.jl:521\u001b[22m\u001b[22m",
      " [3] \u001b[1m(::Knet.#kw##rnnforw)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Any,1}, ::Knet.#rnnforw, ::Knet.RNN, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./<missing>:0\u001b[22m\u001b[22m",
      " [4] \u001b[1m#predict#9\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Int64, ::Function, ::Tuple{Knet.RNN,Knet.KnetArray{Float32,3},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2}}, ::Array{Int64,2}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[9]:6\u001b[22m\u001b[22m",
      " [5] \u001b[1mpredict\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Tuple{Knet.RNN,Knet.KnetArray{Float32,3},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2}}, ::Array{Int64,2}, ::Knet.KnetArray{Float32,3}, ::Knet.KnetArray{Float32,3}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[9]:3\u001b[22m\u001b[22m",
      " [6] \u001b[1m#loss#10\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Any,1}, ::Function, ::Tuple{Knet.RNN,Knet.KnetArray{Float32,3},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2}}, ::Array{Int64,2}, ::Array{Int64,2}, ::Array{Any,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[10]:3\u001b[22m\u001b[22m",
      " [7] \u001b[1mtest\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Tuple{Knet.RNN,Knet.KnetArray{Float32,3},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2},Knet.KnetArray{Float32,2}}, ::Knet.MB\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[12]:5\u001b[22m\u001b[22m",
      " [8] \u001b[1mhyperloss\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Int64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[16]:13\u001b[22m\u001b[22m"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (1024, 128, 3, 0.5)\n"
     ]
    }
   ],
   "source": [
    "model=nothing; knetgc(); hyperloss([3,0,3,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#=\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (1024, 128, 2, 0.5)\n",
    "(:epoch, 1, :trnppl, 9.589466f0, :tstppl, 4.9871907f0)\n",
    "(:epoch, 2, :trnppl, 3.741831f0, :tstppl, 3.9569008f0)\n",
    "(:epoch, 3, :trnppl, 3.111019f0, :tstppl, 3.6438942f0)\n",
    "(:epoch, 4, :trnppl, 2.842651f0, :tstppl, 3.490925f0)\n",
    "(:epoch, 5, :trnppl, 2.6792212f0, :tstppl, 3.406119f0)\n",
    "(:epoch, 6, :trnppl, 2.567202f0, :tstppl, 3.337223f0)\n",
    "(:epoch, 7, :trnppl, 2.484569f0, :tstppl, 3.2894032f0)\n",
    "(:epoch, 8, :trnppl, 2.4184995f0, :tstppl, 3.2509122f0)\n",
    "(:epoch, 9, :trnppl, 2.3658345f0, :tstppl, 3.2483802f0)\n",
    "(:epoch, 10, :trnppl, 2.3232405f0, :tstppl, 3.2095058f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = (1024, 128, 2, 0.5, 2.0069642f0)\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (4096, 128, 1, 0.5)\n",
    "(:epoch, 1, :trnppl, 7.630349f0, :tstppl, 8.994627f0)\n",
    "(:epoch, 2, :trnppl, 3.1094067f0, :tstppl, 3.7359138f0)\n",
    "(:epoch, 3, :trnppl, 2.5211895f0, :tstppl, 3.406949f0)\n",
    "(:epoch, 4, :trnppl, 2.24173f0, :tstppl, 3.3002508f0)\n",
    "(:epoch, 5, :trnppl, 2.0695934f0, :tstppl, 3.2824345f0)\n",
    "(:epoch, 6, :trnppl, 1.9510676f0, :tstppl, 3.2519991f0)\n",
    "(:epoch, 7, :trnppl, 1.8637797f0, :tstppl, 3.2833624f0)\n",
    "(:epoch, 8, :trnppl, 1.7990172f0, :tstppl, 3.2755644f0)\n",
    "(:epoch, 9, :trnppl, 1.7516453f0, :tstppl, 3.3127692f0)\n",
    "(:epoch, 10, :trnppl, 1.7112837f0, :tstppl, 3.3849468f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = (4096, 128, 1, 0.5, 1.5795085f0)\n",
    "1.5795085f0\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (2048, 128, 1, 0.5)\n",
    "(:epoch, 1, :trnppl, 6.897045f0, :tstppl, 4.5084825f0)\n",
    "(:epoch, 2, :trnppl, 3.1653059f0, :tstppl, 3.7902246f0)\n",
    "(:epoch, 3, :trnppl, 2.6861045f0, :tstppl, 3.5171616f0)\n",
    "(:epoch, 4, :trnppl, 2.4583292f0, :tstppl, 3.4100456f0)\n",
    "(:epoch, 5, :trnppl, 2.3147066f0, :tstppl, 3.3488684f0)\n",
    "(:epoch, 6, :trnppl, 2.2171206f0, :tstppl, 3.3439498f0)\n",
    "(:epoch, 7, :trnppl, 2.1446044f0, :tstppl, 3.3066962f0)\n",
    "(:epoch, 8, :trnppl, 2.0894327f0, :tstppl, 3.3102229f0)\n",
    "(:epoch, 9, :trnppl, 2.0460289f0, :tstppl, 3.287103f0)\n",
    "(:epoch, 10, :trnppl, 2.0100765f0, :tstppl, 3.3185039f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = (2048, 128, 1, 0.5, 1.7895192f0)\n",
    "1.7895192f0\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (2896, 128, 1, 0.5)\n",
    "(:epoch, 1, :trnppl, 7.1333003f0, :tstppl, 4.4698453f0)\n",
    "(:epoch, 2, :trnppl, 3.04668f0, :tstppl, 3.719867f0)\n",
    "(:epoch, 3, :trnppl, 2.5365477f0, :tstppl, 3.439777f0)\n",
    "(:epoch, 4, :trnppl, 2.295574f0, :tstppl, 3.3415797f0)\n",
    "(:epoch, 5, :trnppl, 2.1462688f0, :tstppl, 3.27515f0)\n",
    "(:epoch, 6, :trnppl, 2.044871f0, :tstppl, 3.2823677f0)\n",
    "(:epoch, 7, :trnppl, 1.9718276f0, :tstppl, 3.2806444f0)\n",
    "(:epoch, 8, :trnppl, 1.9180799f0, :tstppl, 3.2798069f0)\n",
    "(:epoch, 9, :trnppl, 1.8782355f0, :tstppl, 3.2703993f0)\n",
    "(:epoch, 10, :trnppl, 1.8446114f0, :tstppl, 3.3073356f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = \n",
    "1.6504527f0\n",
    "(2896, 128, 1, 0.5, 1.6504527f0)\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (2048, 128, 1, 0)\n",
    "(:epoch, 1, :trnppl, 17.54582f0, :tstppl, 9.120878f0)\n",
    "(:epoch, 2, :trnppl, 5.3387966f0, :tstppl, 5.298701f0)\n",
    "(:epoch, 3, :trnppl, 3.4404564f0, :tstppl, 4.2200007f0)\n",
    "(:epoch, 4, :trnppl, 2.7447715f0, :tstppl, 3.8503358f0)\n",
    "(:epoch, 5, :trnppl, 2.3742914f0, :tstppl, 3.695858f0)\n",
    "(:epoch, 6, :trnppl, 2.1303215f0, :tstppl, 3.6608467f0)\n",
    "(:epoch, 7, :trnppl, 1.9553958f0, :tstppl, 3.687474f0)\n",
    "(:epoch, 8, :trnppl, 1.8226186f0, :tstppl, 3.7828565f0)\n",
    "(:epoch, 9, :trnppl, 1.7212292f0, :tstppl, 3.8975382f0)\n",
    "(:epoch, 10, :trnppl, 1.6417012f0, :tstppl, 4.0848227f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = \n",
    "1.7484467f0\n",
    "(2048, 128, 1, 0, 1.7484467f0)\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (1024, 128, 2, 0.5)\n",
    "(:epoch, 1, :trnppl, 22.390598f0, :tstppl, 13.189085f0)\n",
    "(:epoch, 2, :trnppl, 7.4823937f0, :tstppl, 5.720241f0)\n",
    "(:epoch, 3, :trnppl, 4.6992884f0, :tstppl, 4.448046f0)\n",
    "(:epoch, 4, :trnppl, 3.7799704f0, :tstppl, 3.972568f0)\n",
    "(:epoch, 5, :trnppl, 3.3514497f0, :tstppl, 3.7654393f0)\n",
    "(:epoch, 6, :trnppl, 3.0903006f0, :tstppl, 3.5971427f0)\n",
    "(:epoch, 7, :trnppl, 2.9079294f0, :tstppl, 3.484381f0)\n",
    "(:epoch, 8, :trnppl, 2.7710462f0, :tstppl, 3.443712f0)\n",
    "(:epoch, 9, :trnppl, 2.663027f0, :tstppl, 3.3361208f0)\n",
    "(:epoch, 10, :trnppl, 2.5764978f0, :tstppl, 3.3100765f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = \n",
    "2.175947f0\n",
    "(1024, 128, 2, 0.5, 2.175947f0)\n",
    "\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT) = (1024, 128, 2, 0)\n",
    "(:epoch, 1, :trnppl, 22.009565f0, :tstppl, 11.428911f0)\n",
    "(:epoch, 2, :trnppl, 6.030965f0, :tstppl, 5.366185f0)\n",
    "(:epoch, 3, :trnppl, 3.614693f0, :tstppl, 4.288177f0)\n",
    "(:epoch, 4, :trnppl, 2.906743f0, :tstppl, 3.948376f0)\n",
    "(:epoch, 5, :trnppl, 2.572164f0, :tstppl, 3.8014898f0)\n",
    "(:epoch, 6, :trnppl, 2.3514614f0, :tstppl, 3.7232788f0)\n",
    "(:epoch, 7, :trnppl, 2.1846726f0, :tstppl, 3.7083063f0)\n",
    "(:epoch, 8, :trnppl, 2.056641f0, :tstppl, 3.7179863f0)\n",
    "(:epoch, 9, :trnppl, 1.9550104f0, :tstppl, 3.768648f0)\n",
    "(:epoch, 10, :trnppl, 1.8722341f0, :tstppl, 3.8625576f0)\n",
    "(HIDDENSIZE, INPUTSIZE, NUMLAYERS, DROPOUT, trnppl) = \n",
    "1.9898754f0\n",
    "\n",
    "=#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"Knet.RNN\", \"1×1×17842176 Knet.KnetArray{Float32,3}\", \"128×128 Knet.KnetArray{Float32,2}\", \"128×2048 Knet.KnetArray{Float32,2}\", \"128×1 Knet.KnetArray{Float32,2}\")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#foo = download(\"http://ai.ku.edu.tr/models/nlp-demos/juliacharlm.jld\")\n",
    "using JLD\n",
    "model = nothing; knetgc()\n",
    "model = load(foo,\"model\")\n",
    "summary.(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(128, 2048, 1, 0.5, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00007f527cfa37c0), Knet.DD(Ptr{Void} @0x00007f527c489fb0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x000001090f000000, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Float32[-0.130193]\n",
       "\n",
       "Float32[-0.110856]\n",
       "\n",
       "Float32[0.224105]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[-0.0971482]\n",
       "\n",
       "Float32[-0.126658]\n",
       "\n",
       "Float32[-0.119905], Float32[0.00395426 0.242381 … 0.004736 0.0848034; 0.00741395 0.0714351 … 0.0387686 0.0708666; … ; -0.151519 0.0125546 … 0.13094 -0.0600793; 0.163786 0.0216087 … -0.110347 -0.167693], Float32[0.11218 0.232794 … 0.0872763 -0.0770994; 0.0390845 -0.219831 … -0.353927 0.0227392; … ; -0.047294 -0.170231 … 0.176739 0.0469917; -0.0778935 -0.077268 … -0.0225408 -0.0718321], Float32[0.740456; 0.329557; … ; -0.184157; -0.247125])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpumodel = (model[1],map(Array,model[2:end])...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    StackTrace:\n",
      "               invoke_fexpr, sv.src, sv.loc_setant, version)\n",
      "        end\n",
      "        fension = vals[idx]\n",
      "        if !effect_free(out_pin, expr.args[2], i, vers, valid_args[2], slot_id, vs)\n",
      "            ssa_func(data, src.codeEstions[l])\n",
      "            if key_common_constant == (maj2_parled && (size - length(val)) > maxline)\n",
      "                throw(ArgumentError(\"invalid constant in different task\"))\n",
      "            end\n",
      "        end\n",
      "    end\n",
      "    pos = count(print_worker_lock(bo)\n",
      "    to_type = abstract_eval_constant(defaultl, default_stream, wlong)\n",
      "    return Tuple{t.storage, invoke_fexpr, error==1)\n",
      "end\n",
      "\n",
      "function delete_value_nopass!(m::Module, ::Type{Val{N}}) where N\n",
      "    # which not checking as Nullex types\n",
      "end\n",
      "\n",
      "end\n",
      "\n",
      "function is_meta_expr_indices(ex, get_replaced)\n",
      "    print_args, print_with_color()\n",
      "    push!(args, labelnum)\n",
      "    if !false\n",
      "        return start(str)\n",
      "    end\n",
      "    esc\n",
      "    pid = options(val, false)\n",
      "    naiss = isa(typ, UnionAll)\n",
      "    str = convert(1, cote)\n",
      "    # Number of outpu\n"
     ]
    }
   ],
   "source": [
    "generate(cpumodel,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knet.RNN\n",
      "  inputSize: Int32 128\n",
      "  hiddenSize: Int32 2048\n",
      "  numLayers: Int32 1\n",
      "  dropout: Float64 0.5\n",
      "  inputMode: Int32 0\n",
      "  direction: Int32 0\n",
      "  mode: Int32 2\n",
      "  algo: Int32 0\n",
      "  dataType: Float32 <: AbstractFloat\n",
      "  rnnDesc: Knet.RD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x00007f527cfa37c0\n",
      "  dropoutDesc: Knet.DD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x00007f527c489fb0\n",
      "    states: Knet.KnetArray{UInt8,1}\n",
      "      ptr: Knet.KnetPtr\n",
      "        ptr: Ptr{Void} Ptr{Void} @0x000001090f000000\n",
      "        len: Int64 1474560\n",
      "        dev: Int64 1\n",
      "        parent: Void nothing\n",
      "      dims: Tuple{Int64}\n",
      "        1: Int64 1474560\n",
      "  dx: Void nothing\n",
      "  dhx: Void nothing\n",
      "  dcx: Void nothing\n"
     ]
    }
   ],
   "source": [
    "dump(cpumodel[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpumodel[1].rnnDesc = cpumodel[1].dropoutDesc = nothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knet.RNN\n",
      "  inputSize: Int32 128\n",
      "  hiddenSize: Int32 2048\n",
      "  numLayers: Int32 1\n",
      "  dropout: Float64 0.5\n",
      "  inputMode: Int32 0\n",
      "  direction: Int32 0\n",
      "  mode: Int32 2\n",
      "  algo: Int32 0\n",
      "  dataType: Float32 <: AbstractFloat\n",
      "  rnnDesc: Void nothing\n",
      "  dropoutDesc: Void nothing\n",
      "  dx: Void nothing\n",
      "  dhx: Void nothing\n",
      "  dcx: Void nothing\n"
     ]
    }
   ],
   "source": [
    "dump(cpumodel[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(\"juliacharlm.jld\",\"model\",cpumodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"/data/scratch/deniz/examples2/dl-tutorial\""
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"/tmp/julianOJPUy\""
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(168, 334, 1, 0.0, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00007f52769a48f0), Knet.DD(Ptr{Void} @0x0000000004f502e0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x0000010913610000, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Knet.KnetArray{Float32,3}(Knet.KnetPtr(Ptr{Void} @0x0000010913c00000, 2693376, 1, nothing), (1, 1, 673344)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x000001090ee20000, 56448, 1, nothing), (168, 84)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x0000010914a30000, 112224, 1, nothing), (84, 334)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x000001090ec01200, 336, 1, nothing), (84, 1)))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shake = load(\"shakespeare.jld\",\"model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "({cd    g inbe ot\n",
      "ll ieahe pr)soelbeo: na, yseoeatlpucd    _iesemrse g f\n",
      "ll art otnp tie( )yra (\n",
      "ae rfacd    \"=n\n",
      "aot tie e6yrst\n",
      "a= tr tie tpsnatxocd    }l\n",
      "..)sp{ ormtna,leo nse =l\n",
      "ttleu S)t g f\n",
      "ll tr)hix,cd    na, `rs, vhrsa: tell (e{ mrs xt\n",
      "o .)t n hr(ynap fesecd    tr hr(e tr iel, (p ien,u Sp mres\n",
      "hR(rssrf: hram)o\n",
      "ra: g oeeucd>Se tie [eatuDDcd  An(u `\n",
      "Te nara inbe arti\n",
      "a= hsrfax, tie mee,\n",
      "a= hsntieocd    Ie=eat: ()ot in, .)txo hr(ysrbe, fsr.esoo otsra={  cd    \"a, fell f\n",
      "oi pr) =r .sentie eatsn\n",
      "ax, trrCcd  vg}gwg#vu Iet)sa hraoeatleoo r(\n",
      "t: o\n",
      "s{cd            \"a, let (e art ,srf arf:cd             vTeey\n",
      "a= rahe: ylrt tie fnllocd           knan=e \n",
      "a tie lenys\n",
      "beocd         B\n",
      "ti .lnhT penso rxest na, irye ien,cd      kn, tint aefoeti: na, tiese{cd          _ip a\n",
      "he senora =snat (e nllucd  _0EIA#vu \"a, iese \n",
      "t \n",
      "o arae \n",
      "a .s\n",
      "=it r)t rmcd     \"a, .s\n",
      "a= i\n",
      "( tr ,\n",
      "e: na, yees n fseToucd     gxll otnp art inotp i\n",
      "(u Seoeeb\n",
      "a= ir(e:cd     _ir) .ensxot fe inbe \n",
      "a i\n",
      "o hrhTo ,eoesbx,:cd     1s \n",
      "a \n"
     ]
    }
   ],
   "source": [
    "generate(shake,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(128, 2048, 1, 0.5, 0, 0, 2, 0, Float32, nothing, nothing, nothing, nothing, nothing), Float32[-0.130193]\n",
       "\n",
       "Float32[-0.110856]\n",
       "\n",
       "Float32[0.224105]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[-0.0971482]\n",
       "\n",
       "Float32[-0.126658]\n",
       "\n",
       "Float32[-0.119905], Float32[0.00395426 0.242381 … 0.004736 0.0848034; 0.00741395 0.0714351 … 0.0387686 0.0708666; … ; -0.151519 0.0125546 … 0.13094 -0.0600793; 0.163786 0.0216087 … -0.110347 -0.167693], Float32[0.11218 0.232794 … 0.0872763 -0.0770994; 0.0390845 -0.219831 … -0.353927 0.0227392; … ; -0.047294 -0.170231 … 0.176739 0.0469917; -0.0778935 -0.077268 … -0.0225408 -0.0718321], Float32[0.740456; 0.329557; … ; -0.184157; -0.247125], [' ', 'e', 't', 'r', 'n', 'i', 'a', 'o', 's', '\\n'  …  'ˢ', '𝕴', '⁉', '𝚗', '🈲', '🔽', '≦', '⋟', '🚊', '𝖆'])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpumodel = (cpumodel..., chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(UInt8[0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x21, 0x27  …  0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d], UInt8[0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d  …  0x2d, 0x33, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d, 0x0e, 0x0d], [' ', 'e', 't', 'o', 'a', 'h', 'n', 's', 'r', 'i'  …  '6', '9', '0', '7', '|', '8', '<', '&', '}', '`'])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "include(Knet.dir(\"data\",\"gutenberg.jl\"))\n",
    "_,_,chars2 = shakespeare()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(168, 334, 1, 0.0, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00007f52769a48f0), Knet.DD(Ptr{Void} @0x0000000004f502e0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x0000010913610000, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Knet.KnetArray{Float32,3}(Knet.KnetPtr(Ptr{Void} @0x0000010913c00000, 2693376, 1, nothing), (1, 1, 673344)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x000001090ee20000, 56448, 1, nothing), (168, 84)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x0000010914a30000, 112224, 1, nothing), (84, 334)), Knet.KnetArray{Float32,2}(Knet.KnetPtr(Ptr{Void} @0x000001090ec01200, 336, 1, nothing), (84, 1)), [' ', 'e', 't', 'o', 'a', 'h', 'n', 's', 'r', 'i'  …  '6', '9', '0', '7', '|', '8', '<', '&', '}', '`'])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shake = (shake..., chars2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MANT IDER. 'Why, you assure me? It may, sir.\n",
      "  QUEEN ELIZABETH. Shallow the matter! unovantis,\n",
      "    When he may give it Harry, that need nothing-\n",
      "    They are the lesser head and writter'd case\n",
      "    And a man of villago'ne, to God fight,\n",
      "    My dead, and stend a more, fright from these men\n",
      "    And revenge within it in his tune,\n",
      "    For unrify may be her; but which you enter\n",
      "    To confirm from my tears.  \n",
      "  Laer. Well, from,  \n",
      "    If you come against their only country\n",
      "    And see, Sir Timon and Lord, then seeks.\n",
      "  GLOUCESTER. The King will espare your ladous doth mother,\n",
      "    And the rreast but too good scringes or mine\n",
      "    To leave on their grief should allock'd his help,\n",
      "    Either-heaven defended. Pray to thy bequir's\n",
      "    The forwarding what they are resolvied.\n",
      "    See him all this merry meat.\n",
      "  FABIAN. Whele I may may be hold? Is't plain sighs; sit. Thou weep\n",
      "    A crown thou point Nirsh; betwixt in time on the means so\n",
      "    For justice could win the throne of eagl\n"
     ]
    }
   ],
   "source": [
    "generate(shake,1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(\"Knet.RNN\", \"1×1×673344 Knet.KnetArray{Float32,3}\", \"168×84 Knet.KnetArray{Float32,2}\", \"84×334 Knet.KnetArray{Float32,2}\", \"84×1 Knet.KnetArray{Float32,2}\", \"84-element Array{Char,1}\")"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary.(shake)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(168, 334, 1, 0.0, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00007f52769a48f0), Knet.DD(Ptr{Void} @0x0000000004f502e0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x0000010913610000, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Float32[-0.412303]\n",
       "\n",
       "Float32[0.117722]\n",
       "\n",
       "Float32[-0.0357888]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[0.0439149]\n",
       "\n",
       "Float32[0.101626]\n",
       "\n",
       "Float32[0.0703143], Float32[0.143123 -0.283479 … 0.236566 -0.0708616; 0.220493 0.191383 … 0.143039 -0.0722817; … ; -0.0330645 -0.653156 … 0.291897 -0.186715; 0.257983 -0.29303 … 0.204898 -0.0792709], Float32[-0.104172 0.10832 … -0.790624 0.168949; -0.0743707 0.155518 … 1.22177 0.560434; … ; 0.0192348 0.121797 … -0.225259 0.012597; 0.00482724 0.111904 … -0.110344 0.0327026], Float32[0.0216497; 0.0268085; … ; -0.0352384; -0.0370137], [' ', 'e', 't', 'o', 'a', 'h', 'n', 's', 'r', 'i'  …  '6', '9', '0', '7', '|', '8', '<', '&', '}', '`'])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpushake = (shake[1],map(Array,shake[2:end])...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knet.RNN\n",
      "  inputSize: Int32 168\n",
      "  hiddenSize: Int32 334\n",
      "  numLayers: Int32 1\n",
      "  dropout: Float64 0.0\n",
      "  inputMode: Int32 0\n",
      "  direction: Int32 0\n",
      "  mode: Int32 2\n",
      "  algo: Int32 0\n",
      "  dataType: Float32 <: AbstractFloat\n",
      "  rnnDesc: Knet.RD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x00007f52769a48f0\n",
      "  dropoutDesc: Knet.DD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x0000000004f502e0\n",
      "    states: Knet.KnetArray{UInt8,1}\n",
      "      ptr: Knet.KnetPtr\n",
      "        ptr: Ptr{Void} Ptr{Void} @0x0000010913610000\n",
      "        len: Int64 1474560\n",
      "        dev: Int64 1\n",
      "        parent: Void nothing\n",
      "      dims: Tuple{Int64}\n",
      "        1: Int64 1474560\n",
      "  dx: Void nothing\n",
      "  dhx: Void nothing\n",
      "  dcx: Void nothing\n"
     ]
    }
   ],
   "source": [
    "dump(cpushake[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "cpushake[1].rnnDesc = cpushake[1].dropoutDesc = nothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(\"shakespeare.jld\",\"model\",cpushake)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      "100 2813k  100 2813k    0     0  2333k      0  0:00:01  0:00:01 --:--:-- 2333k\n",
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      " 91 69.1M   91 63.2M    0     0  5473k      0  0:00:12  0:00:11  0:00:01 5618k"
     ]
    },
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mSystemError: opening file foo1: No such file or directory\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mSystemError: opening file foo1: No such file or directory\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1m#systemerror#44\u001b[22m\u001b[22m at \u001b[1m./error.jl:64\u001b[22m\u001b[22m [inlined]",
      " [2] \u001b[1msystemerror\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::Bool\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./error.jl:64\u001b[22m\u001b[22m",
      " [3] \u001b[1mopen\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::Bool, ::Bool, ::Bool, ::Bool, ::Bool\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./iostream.jl:104\u001b[22m\u001b[22m",
      " [4] \u001b[1m#load#27\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Any,1}, ::Function, ::FileIO.File{FileIO.DataFormat{:UNKNOWN}}, ::String, ::Vararg{String,N} where N\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/v0.6/FileIO/src/loadsave.jl:168\u001b[22m\u001b[22m",
      " [5] \u001b[1mload\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::FileIO.File{FileIO.DataFormat{:UNKNOWN}}, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/v0.6/FileIO/src/loadsave.jl:167\u001b[22m\u001b[22m",
      " [6] \u001b[1m#load#13\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Any,1}, ::Function, ::String, ::String, ::Vararg{String,N} where N\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/v0.6/FileIO/src/loadsave.jl:113\u001b[22m\u001b[22m",
      " [7] \u001b[1mload\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/data/scratch/deniz/v0.6/FileIO/src/loadsave.jl:113\u001b[22m\u001b[22m"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100 69.1M  100 69.1M    0     0  5728k      0  0:00:12  0:00:12 --:--:-- 6895k\n"
     ]
    }
   ],
   "source": [
    "foo1 = download(\"http://people.csail.mit.edu/deniz/models/nlp-demos/shakespeare.jld\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(128, 2048, 1, 0.5, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00000000018273d0), Knet.DD(Ptr{Void} @0x00007f52734da4c0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x0000010913e91a00, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Float32[-0.130193]\n",
       "\n",
       "Float32[-0.110856]\n",
       "\n",
       "Float32[0.224105]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[-0.0971482]\n",
       "\n",
       "Float32[-0.126658]\n",
       "\n",
       "Float32[-0.119905], Float32[0.00395426 0.242381 … 0.004736 0.0848034; 0.00741395 0.0714351 … 0.0387686 0.0708666; … ; -0.151519 0.0125546 … 0.13094 -0.0600793; 0.163786 0.0216087 … -0.110347 -0.167693], Float32[0.11218 0.232794 … 0.0872763 -0.0770994; 0.0390845 -0.219831 … -0.353927 0.0227392; … ; -0.047294 -0.170231 … 0.176739 0.0469917; -0.0778935 -0.077268 … -0.0225408 -0.0718321], Float32[0.740456; 0.329557; … ; -0.184157; -0.247125])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m1 = load(foo1,\"model\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "en Surreys, how?\n",
      "  MESSALA. Ay, nor I am merry; that's lury.\n",
      "  CELIA. Thy hair is a man, his princ\n"
     ]
    }
   ],
   "source": [
    "generate(m1,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "ename": "LoadError",
     "evalue": "\u001b[91mBoundsError: attempt to access 128×128 Array{Float32,2} at index [Base.Slice(Base.OneTo(128)), [0]]\u001b[39m",
     "output_type": "error",
     "traceback": [
      "\u001b[91mBoundsError: attempt to access 128×128 Array{Float32,2} at index [Base.Slice(Base.OneTo(128)), [0]]\u001b[39m",
      "",
      "Stacktrace:",
      " [1] \u001b[1mthrow_boundserror\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float32,2}, ::Tuple{Base.Slice{Base.OneTo{Int64}},Array{Int64,1}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:434\u001b[22m\u001b[22m",
      " [2] \u001b[1mcheckbounds\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:362\u001b[22m\u001b[22m [inlined]",
      " [3] \u001b[1mmacro expansion\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:494\u001b[22m\u001b[22m [inlined]",
      " [4] \u001b[1m_getindex\u001b[22m\u001b[22m at \u001b[1m./multidimensional.jl:491\u001b[22m\u001b[22m [inlined]",
      " [5] \u001b[1mgetindex\u001b[22m\u001b[22m at \u001b[1m./abstractarray.jl:883\u001b[22m\u001b[22m [inlined]",
      " [6] \u001b[1m#predict#9\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Int64, ::Function, ::Tuple{Knet.RNN,Array{Float32,3},Array{Float32,2},Array{Float32,2},Array{Float32,2}}, ::Array{Int64,1}, ::Void, ::Void\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[9]:4\u001b[22m\u001b[22m",
      " [7] \u001b[1mgenerate\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Tuple{Knet.RNN,Array{Float32,3},Array{Float32,2},Array{Float32,2},Array{Float32,2}}, ::Int64\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./In[37]:11\u001b[22m\u001b[22m"
     ]
    }
   ],
   "source": [
    "generate(m2,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(128, 2048, 1, 0.5, 0, 0, 2, 0, Float32, nothing, nothing, nothing, nothing, nothing), Float32[-0.130193]\n",
       "\n",
       "Float32[-0.110856]\n",
       "\n",
       "Float32[0.224105]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[-0.0971482]\n",
       "\n",
       "Float32[-0.126658]\n",
       "\n",
       "Float32[-0.119905], Float32[0.00395426 0.242381 … 0.004736 0.0848034; 0.00741395 0.0714351 … 0.0387686 0.0708666; … ; -0.151519 0.0125546 … 0.13094 -0.0600793; 0.163786 0.0216087 … -0.110347 -0.167693], Float32[0.11218 0.232794 … 0.0872763 -0.0770994; 0.0390845 -0.219831 … -0.353927 0.0227392; … ; -0.047294 -0.170231 … 0.176739 0.0469917; -0.0778935 -0.077268 … -0.0225408 -0.0718321], Float32[0.740456; 0.329557; … ; -0.184157; -0.247125], [' ', 'e', 't', 'r', 'n', 'i', 'a', 'o', 's', '\\n'  …  'ˢ', '𝕴', '⁉', '𝚗', '🈲', '🔽', '≦', '⋟', '🚊', '𝖆'])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cpumodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(\"juliacharlm.jld\",\"model\",cpumodel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\n",
      " 93 69.1M   93 64.3M    0     0  7295k      0  0:00:09  0:00:09 --:--:-- 7242k"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(Knet.RNN(128, 2048, 1, 0.5, 0, 0, 2, 0, Float32, Knet.RD(Ptr{Void} @0x00007f5277618250), Knet.DD(Ptr{Void} @0x000000000192cff0, Knet.KnetArray{UInt8,1}(Knet.KnetPtr(Ptr{Void} @0x0000010913610000, 1474560, 1, nothing), (1474560,))), nothing, nothing, nothing), Float32[-0.130193]\n",
       "\n",
       "Float32[-0.110856]\n",
       "\n",
       "Float32[0.224105]\n",
       "\n",
       "...\n",
       "\n",
       "Float32[-0.0971482]\n",
       "\n",
       "Float32[-0.126658]\n",
       "\n",
       "Float32[-0.119905], Float32[0.00395426 0.242381 … 0.004736 0.0848034; 0.00741395 0.0714351 … 0.0387686 0.0708666; … ; -0.151519 0.0125546 … 0.13094 -0.0600793; 0.163786 0.0216087 … -0.110347 -0.167693], Float32[0.11218 0.232794 … 0.0872763 -0.0770994; 0.0390845 -0.219831 … -0.353927 0.0227392; … ; -0.047294 -0.170231 … 0.176739 0.0469917; -0.0778935 -0.077268 … -0.0225408 -0.0718321], Float32[0.740456; 0.329557; … ; -0.184157; -0.247125], [' ', 'e', 't', 'r', 'n', 'i', 'a', 'o', 's', '\\n'  …  'ˢ', '𝕴', '⁉', '𝚗', '🈲', '🔽', '≦', '⋟', '🚊', '𝖆'])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "100 69.1M  100 69.1M    0     0  7533k      0  0:00:09  0:00:09 --:--:-- 7965k\n"
     ]
    }
   ],
   "source": [
    "foo2 = download(\"http://people.csail.mit.edu/deniz/models/nlp-demos/juliacharlm.jld\")\n",
    "m2 = load(foo2,\"model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        # fixing mode\n",
      "        containspind(C)\n",
      "    end\n",
      "    ogd_println = filter(0 -> delete!(spp[i], \n"
     ]
    }
   ],
   "source": [
    "generate(m2,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Knet.RNN\n",
      "  inputSize: Int32 128\n",
      "  hiddenSize: Int32 2048\n",
      "  numLayers: Int32 1\n",
      "  dropout: Float64 0.5\n",
      "  inputMode: Int32 0\n",
      "  direction: Int32 0\n",
      "  mode: Int32 2\n",
      "  algo: Int32 0\n",
      "  dataType: Float32 <: AbstractFloat\n",
      "  rnnDesc: Knet.RD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x00007f5277618250\n",
      "  dropoutDesc: Knet.DD\n",
      "    ptr: Ptr{Void} Ptr{Void} @0x000000000192cff0\n",
      "    states: Knet.KnetArray{UInt8,1}\n",
      "      ptr: Knet.KnetPtr\n",
      "        ptr: Ptr{Void} Ptr{Void} @0x0000010913610000\n",
      "        len: Int64 1474560\n",
      "        dev: Int64 1\n",
      "        parent: Void nothing\n",
      "      dims: Tuple{Int64}\n",
      "        1: Int64 1474560\n",
      "  dx: Void nothing\n",
      "  dhx: Void nothing\n",
      "  dcx: Void nothing\n"
     ]
    }
   ],
   "source": [
    "dump(m2[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.3",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
